front_end/panels/ai_assistance/components/ChatView.ts (1)

1446-1447: Fix event and target typing for textarea input.

This is a textarea, not an input, and the handler is an input event, not a keyboard event. Avoids incorrect casts and TS friction.

-        @input=${(event: KeyboardEvent) => onTextInputChange((event.target as HTMLInputElement).value)}
+        @input=${(event: Event) => onTextInputChange((event.target as HTMLTextAreaElement).value)}

front_end/panels/ai_chat/core/GraphHelpers.ts (2)

44-47: Use explicit default agent type instead of empty string.

Passing '' relies on downstream fallback; use 'default' for determinism.

 export function getAgentToolsFromState(state: AgentState): ReturnType<typeof getTools> {
   // Use the helper from BaseOrchestratorAgent to get the pre-filtered list
-  return BaseOrchestratorAgent.getAgentTools(state.selectedAgentType ?? ''); // Pass agentType or empty string
+  return BaseOrchestratorAgent.getAgentTools(state.selectedAgentType ?? 'default');
 }

---

18-28: Migrate remaining synchronous createSystemPrompt calls to async version
– In front_end/panels/ai_chat/tools/HTMLToMarkdownTool.ts replace the call at line 104 (this.createSystemPrompt()) with an awaited this.createSystemPromptAsync(state) (and mark the enclosing method as async), then deprecate or remove the private sync createSystemPrompt implementation at line 158.

front_end/panels/ai_chat/evaluation/remote/EvaluationAgent.ts (3)

402-416: Fallback to action_agent uses wrong input schema.

action_agent requires { objective, reasoning } per its schema; passing { task, reasoning } will fail validation.

Apply:

-                { 
-                  task: `Navigate to ${params.url}`,
-                  reasoning: 'Navigation required for evaluation'
-                },
+                {
+                  objective: `Navigate to ${params.url}`,
+                  reasoning: 'Navigation required for evaluation'
+                },

---

438-446: Chat model override is ignored.

You merge params.model into mergedInput but executeChatEvaluation always takes model from localStorage/default, discarding overrides.

Apply:

-        // Use the current model from localStorage (no override)
-        let modelName = localStorage.getItem('ai_chat_model_selection');
-        if (!modelName) {
-          // Default model
-          modelName = 'gpt-4o';
-        }
+        // Prefer explicit input override, then local selection, then default
+        let modelName =
+          (input?.main_model as string | undefined) ||
+          localStorage.getItem('ai_chat_model_selection') ||
+          'gpt-4o';

Also applies to: 705-719

---

281-290: Disconnect on failed secret verification.

After sending a negative verification, reject and disconnect to avoid dangling connections.

Apply:

       if (!verified) {
         logger.error('Secret key verification failed - keys do not match');
         // Reject auth promise immediately since we know auth will fail
         if (this.authReject) {
           this.authReject(new Error('Secret key verification failed - keys do not match'));
           this.authResolve = null;
           this.authReject = null;
         }
+        this.disconnect();
       }

front_end/panels/ai_chat/core/StateGraph.ts (1)

72-99: Tracing calls should not be able to break graph execution.

createObservation calls are awaited without try/catch; a tracing backend hiccup will abort the graph.

Apply:

-      let spanId: string | undefined;
       const spanStartTime = new Date();
 
       if (tracingContext?.traceId) {
-        spanId = `span-${currentNodeName}-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`;
-        
-        // Create a start event for the node
-        await this.tracingProvider.createObservation({
+        // Create a start event for the node
+        try {
+          await this.tracingProvider.createObservation({
             id: `event-start-${currentNodeName}-${Date.now()}`,
             name: `Start Node: ${currentNodeName}`,
             type: 'event',
             startTime: spanStartTime,
             parentObservationId: tracingContext.parentObservationId,
             input: {
               nodeType: currentNodeName,
               graphName: this.name,
               step,
               messagesCount: currentState.messages?.length || 0
             },
             metadata: {
               nodeType: currentNodeName,
               graphName: this.name,
               step,
               phase: 'start'
             }
-        }, tracingContext.traceId);
+          }, tracingContext.traceId);
+        } catch (e) {
+          logger.warn('Tracing start event failed:', e);
+        }
       }
@@
-        if (spanId && tracingContext?.traceId) {
-          await this.tracingProvider.createObservation({
+        if (tracingContext?.traceId) {
+          try {
+            await this.tracingProvider.createObservation({
               id: `event-complete-${currentNodeName}-${Date.now()}`,
               name: `Complete Node: ${currentNodeName}`,
               type: 'event',
               startTime: new Date(),
               parentObservationId: tracingContext.parentObservationId,
               output: { 
                 success: true,
                 duration: Date.now() - spanStartTime.getTime(),
                 messagesAfter: currentState.messages?.length || 0
               },
               metadata: {
                 nodeType: currentNodeName,
                 graphName: this.name,
                 step,
                 phase: 'complete'
               }
-          }, tracingContext.traceId);
+            }, tracingContext.traceId);
+          } catch (e) {
+            logger.warn('Tracing completion event failed:', e);
+          }
         }
@@
-        if (spanId && tracingContext?.traceId) {
-          await this.tracingProvider.createObservation({
+        if (tracingContext?.traceId) {
+          try {
+            await this.tracingProvider.createObservation({
               id: `event-error-${currentNodeName}-${Date.now()}`,
               name: `Error Node: ${currentNodeName}`,
               type: 'event',
               startTime: new Date(),
               parentObservationId: tracingContext.parentObservationId,
               error: error instanceof Error ? error.message : String(error),
               metadata: {
                 nodeType: currentNodeName,
                 graphName: this.name,
                 step,
                 phase: 'error'
               }
-          }, tracingContext.traceId);
+            }, tracingContext.traceId);
+          } catch (e) {
+            logger.warn('Tracing error event failed:', e);
+          }
         }

Also applies to: 105-124, 132-147

front_end/panels/ai_chat/agent_framework/implementation/ConfiguredAgents.ts (2)

620-641: Action Verification Agent references unregistered tools.

inspect_element and get_console_logs are not registered/imported; tool calls will fail at runtime.

Either register these tools or remove them from the agent’s tool list and prompt. Minimal fix:

-## Tools to Use
-- get_page_content: Examine the updated page structure
-- search_content: Look for specific text indicating success/failure
-- inspect_element: Check properties of specific elements
-- get_console_logs: Check for errors or success messages in the console
-- schema_based_extractor: Extract structured data to verify expected outcomes
-- take_screenshot: Capture the current page state
+## Tools to Use
+- get_page_content: Examine the updated page structure
+- search_content: Look for specific text indicating success/failure
+- schema_based_extractor: Extract structured data to verify expected outcomes
+- take_screenshot: Capture the current page state
@@
-    tools: [
-      'search_content',
-      'inspect_element',
-      'get_console_logs',
-      'schema_based_extractor',
-      'take_screenshot'
-    ],
+    tools: ['search_content', 'schema_based_extractor', 'take_screenshot'],

If you intended to support those tools, I can add the implementations and registrations instead.

---

1421-1423: E‑commerce agent prompt vs tools mismatch.

System prompt instructs use of schema_based_extractor and search_content, but they’re not in tools[]. Add them.

Apply:

-    tools: [
-      'navigate_url',
-      'get_page_content',
-    ],
+    tools: [
+      'navigate_url',
+      'get_page_content',
+      'schema_based_extractor',
+      'search_content',
+    ],

front_end/panels/ai_chat/tools/CombinedExtractionTool.ts (2)

55-76: Fix tool schema: required keys mismatch and missing extractionInstruction.

Schema marks schema and markdownResponse as required, but the args interface makes them optional. Also, extractionInstruction is used but not declared in properties. This will cause validation failures or dropped fields.

Apply:

   schema = {
     type: 'object',
     properties: {
       url: {
         type: 'string',
         description: 'URL to navigate to'
       },
       schema: {
         type: 'object',
         description: 'JSON Schema definition of the data to extract'
       },
       markdownResponse: {
         type: 'boolean',
         description: 'If true, converts the page content to Markdown'
       },
       reasoning: {
         type: 'string',
         description: 'Reasoning for the action displayed to the user'
       },
+      extractionInstruction: {
+        type: 'string',
+        description: 'Optional instruction to guide extraction/markdown conversion'
+      },
     },
-    required: ['url', 'reasoning', 'schema', 'markdownResponse']
+    required: ['url', 'reasoning']
   };

---

81-93: Validate ctx when sub-tools need LLM context; fail fast with a clear message.

Both SchemaBasedExtractorTool and HTMLToMarkdownTool now consume ctx; without provider/model this can throw at runtime.

Minimal guard:

   async execute(args: CombinedExtractionArgs, ctx?: LLMContext): Promise<CombinedExtractionResult | ErrorResult> {
     logger.info('Executing with args', { args });
     const { url, schema, markdownResponse, reasoning, extractionInstruction } = args;
     const agentService = AgentService.getInstance();
     const apiKey = agentService.getApiKey();

     if (!apiKey) {
       return {
         success: false,
         url,
         error: 'API key not configured'
       };
     }
+
+    // If either operation may rely on LLM, ensure ctx is present
+    if ((schema || markdownResponse) && (!ctx?.provider || !ctx?.model)) {
+      return {
+        success: false,
+        url,
+        error: 'LLM context missing provider/model'
+      };
+    }

Also applies to: 131-141, 149-157

front_end/panels/ai_chat/tools/CritiqueTool.ts (2)

51-51: Fix user-facing typo

“finalresponse” → “final response”.

-  description = 'Evaluates if finalresponse satisfies the user\'s requirements and provides feedback if needed.';
+  description = 'Evaluates if final response satisfies the user\'s requirements and provides feedback if needed.';

---

82-89: Align input validation with schema (also require reasoning)

Schema marks reasoning as required; validate it too and clarify error.

-    if (!userInput || !finalResponse) {
+    if (!userInput || !finalResponse || !reasoning) {
       return {
         satisfiesCriteria: false,
         success: false,
-        error: 'Both user input and final response must be provided.'
+        error: 'userInput, finalResponse, and reasoning must be provided.'
       };
     }

front_end/panels/ai_chat/tools/SchemaBasedExtractorTool.ts (1)

77-91: Fail fast if LLM context is missing

Multiple inner calls throw later; validate up-front for clearer errors.

 async execute(args: SchemaExtractionArgs, ctx?: LLMContext): Promise<SchemaExtractionResult> {
   logger.debug('Executing with args', args);
 
   const { schema, instruction, reasoning } = args;
   const agentService = AgentService.getInstance();
   const apiKey = agentService.getApiKey();
 
   if (!apiKey) {
     return {
       success: false,
       data: null,
       error: 'API key not configured'
     };
   }
+  if (!ctx?.provider || !(ctx.nanoModel || ctx.model)) {
+    return {
+      success: false,
+      data: null,
+      error: 'Missing LLM context (provider/model)'
+    };
+  }

front_end/panels/ai_chat/tools/SequentialThinkingTool.ts (1)

225-229: Possible invalid image payload across providers (data URL vs http URL).

If screenshot is base64 without data: prefix, many providers will reject image_url.url. Normalize before send.

-      images: [{
-        type: 'image_url',
-        data: visualContext.screenshot
-      }]
+      images: [{
+        type: 'image_url',
+        data: visualContext.screenshot
+      }]

And in getGroundedAnalysis:

-        const validImages = prompt.images.filter(img => !!img?.data && img.data !== 'no-screenshot-available');
+        const toUrl = (s: string) => s.startsWith('http') || s.startsWith('data:')
+          ? s
+          : `data:image/png;base64,${s}`;
+        const validImages = prompt.images.filter(img => !!img?.data && img.data !== 'no-screenshot-available');
...
-            ...validImages.map(img => ({
+            ...validImages.map(img => ({
               type: 'image_url' as const,
-              image_url: { url: img.data }
+              image_url: { url: toUrl(img.data) }
             }))

Also applies to: 255-265

front_end/panels/ai_chat/ui/AIChatPanel.test.ts (2)

204-221: Assertion uses Map.get(...) instead of window.localStorage.getItem(...).

mockLocalStorage.get(...) returns undefined, not null. The test will fail intermittently.

-      assert.strictEqual(mockLocalStorage.get('ai_chat_mini_model'), null);
+      assert.isNull(window.localStorage.getItem('ai_chat_mini_model'));

---

350-373: Cannot assign to private fields (#agentService, #canSendMessages) from tests.

(panel as any)['#agentService'] and ['#canSendMessages'] won’t patch real private fields; the test won’t mock sendMessage as intended.

Two fixes; pick one:

A) Add a testing hook in AIChatPanel (preferred).

// In AIChatPanel.ts
+ /** TEST-ONLY: inject a mock AgentService */
+ static setAgentServiceForTesting(svc: Pick<AgentService, 'sendMessage'>) {
+   AIChatPanel.instance()['#agentService'] = svc as any;
+ }

And in the test:

-      (panel as any)['#agentService'] = mockAgentService;
-      (panel as any)['#canSendMessages'] = true;
+      AIChatPanel.setAgentServiceForTesting(mockAgentService as any);

B) Monkey-patch AgentService.getInstance in the test.

-      const mockAgentService = { sendMessage: async () => Promise.resolve(), };
-      (panel as any)['#agentService'] = mockAgentService;
-      (panel as any)['#canSendMessages'] = true;
+      const mockAgentService = { sendMessage: async () => Promise.resolve() } as any;
+      const originalGetInstance = AgentService.getInstance;
+      // @ts-ignore
+      AgentService.getInstance = () => mockAgentService;
+      try {
+        await panel.sendMessage('test message');
+      } finally {
+        AgentService.getInstance = originalGetInstance;
+      }

front_end/panels/ai_chat/agent_framework/ConfigurableAgentTool.ts (2)

167-191: String interpolation bug in logger.

Template string is in single quotes; logs ${name} literally.

-        logger.info('Registered and instantiated tool: ${name}');
+        logger.info(`Registered and instantiated tool: ${name}`);

---

386-413: Expose requiresApiKey() on AgentService and use it for gating
Add a public requiresApiKey() wrapper around the private #doesCurrentConfigRequireApiKey() in core/AgentService.ts and update ConfigurableAgentTool.ts to only hard‐fail when that returns true:

 // ConfigurableAgentTool.ts
   const apiKey = agentService.getApiKey();
-  if (!apiKey) {
+  const requiresKey = agentService.requiresApiKey();
+  if (requiresKey && !apiKey) {
     const errorResult = this.createErrorResult(`API key not configured for ${this.name}`, [], 'error');
     …
     return { …errorResult, agentSession: errorSession };
   }

front_end/panels/ai_chat/core/ConfigurableGraph.ts (1)

32-43: Add provider/modelName to base graph configurations
front_end/panels/ai_chat/core/GraphConfigs.ts (lines 15–20) defines agent/toolExecutor nodes without provider or modelName, causing createAgentGraphFromConfig to fail.

front_end/panels/ai_chat/tools/StreamlinedSchemaExtractorTool.ts (1)

271-275: Bug: LLM JSON not parsed (breaks URL resolution and retry paths)

llmResponse.text is returned as a string; downstream resolvers expect objects. Parse before returning.

-        const result = llmResponse.text;
-        
-        logger.debug(`JSON extraction successful on attempt ${attempt}`);
-        return result;
+        const raw = llmResponse.text;
+        const parsed = typeof raw === 'string' ? JSON.parse(raw) : raw;
+        logger.debug(`JSON extraction successful on attempt ${attempt}`);
+        return parsed;

-      const result = llmResponse.text;
-      
-      return result;
+      const raw = llmResponse.text;
+      const parsed = typeof raw === 'string' ? JSON.parse(raw) : raw;
+      return parsed;

Also applies to: 414-416

front_end/panels/ai_chat/ui/AIChatPanel.ts (2)

49-72: Stop logging secrets; gate the localStorage monkey-patch behind an explicit debug flag

The storage monitor currently logs value previews/lengths for keys that are likely to contain API keys/tokens and is always active. This risks credential leakage to logs and alters global behavior in production.

Apply masking and an opt-in flag:

   private setupStorageMonitoring(): void {
     // Monitor setItem operations
-    localStorage.setItem = (key: string, value: string) => {
-      if (key.includes('openrouter') || key.includes('ai_chat')) {
+    localStorage.setItem = (key: string, value: string) => {
+      const shouldLog = (key.includes('openrouter') || key.includes('ai_chat')) &&
+        localStorage.getItem('ai_chat_debug_storage_monitor') === 'true';
+      if (shouldLog) {
+        const isSensitive = /api_key|secret|token|auth/i.test(key);
         logger.debug(`=== LOCALSTORAGE SET ===`);
         logger.debug(`Key: ${key}`);
-        logger.debug(`Value exists: ${!!value}`);
-        logger.debug(`Value length: ${value?.length || 0}`);
-        logger.debug(`Value preview: ${value?.substring(0, 50) + (value?.length > 50 ? '...' : '') || 'null'}`);
+        logger.debug(`Value exists: ${!!value}`);
+        if (!isSensitive) {
+          logger.debug(`Value length: ${value?.length || 0}`);
+          logger.debug(`Value preview: ${value?.substring(0, 50) + (value?.length > 50 ? '...' : '') || 'null'}`);
+        } else {
+          logger.debug(`Value: <redacted>`);
+        }
         logger.debug(`Timestamp: ${new Date().toISOString()}`);
       }
       return this.originalSetItem(key, value);
     };
     
     // Monitor removeItem operations
-    localStorage.removeItem = (key: string) => {
-      if (key.includes('openrouter') || key.includes('ai_chat')) {
+    localStorage.removeItem = (key: string) => {
+      const shouldLog = (key.includes('openrouter') || key.includes('ai_chat')) &&
+        localStorage.getItem('ai_chat_debug_storage_monitor') === 'true';
+      if (shouldLog) {
         logger.debug(`=== LOCALSTORAGE REMOVE ===`);
         logger.debug(`Key: ${key}`);
         logger.debug(`Timestamp: ${new Date().toISOString()}`);
       }
       return this.originalRemoveItem(key);
     };
   }

Additionally consider only constructing StorageMonitor when the same flag is set to 'true'.

---

644-651: Bug: openrouter custom models don’t update MODEL_OPTIONS when openrouter is selected

The current-provider branch omits openrouter.

-    if ((currentProvider === 'openai' && modelType === 'openai') || 
-        (currentProvider === 'litellm' && modelType === 'litellm') ||
-        (currentProvider === 'groq' && modelType === 'groq')) {
+    if ((currentProvider === 'openai' && modelType === 'openai') || 
+        (currentProvider === 'litellm' && modelType === 'litellm') ||
+        (currentProvider === 'groq' && modelType === 'groq') ||
+        (currentProvider === 'openrouter' && modelType === 'openrouter')) {
       MODEL_OPTIONS = [...MODEL_OPTIONS, newOption];
       localStorage.setItem('ai_chat_model_options', JSON.stringify(MODEL_OPTIONS));
     }

front_end/panels/ai_chat/core/AgentService.ts (1)

466-473: Type mismatch: adding non-declared error field to ModelChatMessage.

Either extend the type or intersect when constructing the error message to satisfy TS.

-      const errorMessage: ModelChatMessage = {
+      const errorMessage: ModelChatMessage & { error: string } = {
         entity: ChatMessageEntity.MODEL,
         action: 'final',
         answer: error instanceof Error ? error.message : String(error),
         isFinalAnswer: true,
         error: error instanceof Error ? error.message : String(error),
       };

Alternatively, add error?: string to ModelChatMessage in ChatTypes.ts.

front_end/panels/ai_chat/agent_framework/AgentRunner.ts (2)

161-184: Deep-sanitize tool results and redact secrets before LLM/log storage

Current sanitization is shallow and may leak large base64 blobs or secrets (apiKey, Authorization, cookies) via nested objects.

Apply this diff to replace the method with a recursive, size-aware redactor:

   private static sanitizeToolResultForText(toolResultData: any): any {
-    if (typeof toolResultData !== 'object' || toolResultData === null) {
-      return toolResultData;
-    }
-
-    // Create a shallow copy
-    const sanitized = { ...toolResultData };
-
-    // Remove fields that shouldn't be sent to LLM
-    const fieldsToRemove = [
-      'imageData',    // Prevents token waste from base64 strings
-      'success',      // LLM should infer success from error presence
-      'dataUrl',      // Legacy image field if any
-      'agentSession', // Avoid sending session data to LLM
-    ];
-
-    fieldsToRemove.forEach(field => {
-      if (sanitized.hasOwnProperty(field)) {
-        delete sanitized[field];
-      }
-    });
-
-    return sanitized;
+    if (toolResultData === null || typeof toolResultData !== 'object') {
+      return toolResultData;
+    }
+
+    const redactedKeys = new Set([
+      'imageData', 'dataUrl', 'screenshot', 'blob', 'file', 'binary', 'base64',
+      'agentSession', 'headers', 'cookies', 'setCookie',
+      'apiKey', 'apikey', 'token', 'accessToken', 'authorization', 'authorizationHeader',
+    ]);
+
+    const visited = new WeakSet<object>();
+    const MAX_DEPTH = 4;
+    const MAX_STRING = 10_000;
+
+    const redact = (val: any, depth: number): any => {
+      if (val === null || typeof val !== 'object') {
+        if (typeof val === 'string' && val.length > MAX_STRING) {
+          return `[omitted ${val.length} chars]`;
+        }
+        return val;
+      }
+      if (visited.has(val) || depth > MAX_DEPTH) {
+        return '[omitted]';
+      }
+      visited.add(val);
+
+      if (Array.isArray(val)) {
+        return val.map(v => redact(v, depth + 1));
+      }
+
+      const out: Record<string, any> = {};
+      for (const [k, v] of Object.entries(val)) {
+        if (redactedKeys.has(k)) {
+          continue; // drop entirely
+        }
+        out[k] = redact(v, depth + 1);
+      }
+      return out;
+    };
+
+    return redact(toolResultData, 0);
   }

---

1072-1081: Store sanitized results in messages to prevent base64 bloat/PII retention

Use the sanitizer for both the LLM-facing ToolResultMessage and the session timeline message.

Apply this diff:

           const toolResultMessage: ToolResultMessage = {
             entity: ChatMessageEntity.TOOL_RESULT,
             toolName,
             resultText: toolResultText,
             isError: toolIsError,
             toolCallId, // Link back to the tool call for OpenAI format
             ...(toolIsError && { error: toolResultText }), // Include raw error message if error occurred
-            ...(toolResultData && { resultData: toolResultData }), // Include structured result data
+            ...(toolResultData && { resultData: AgentRunner.sanitizeToolResultForText(toolResultData) }), // Include sanitized data
             ...(imageData && { imageData: imageData }) // Include image data for multimodal LLM responses
           };

           addSessionMessage({
             type: 'tool_result',
             content: {
               type: 'tool_result',
               toolCallId,
               toolName,
               success: !toolIsError,
-              result: toolResultData,
+              result: AgentRunner.sanitizeToolResultForText(toolResultData),
               error: toolIsError ? toolResultText : undefined
             }
           });

Also applies to: 1093-1103